package edu.hit.crawler.test;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;

import org.apache.mina.filter.buffer.BufferedWriteFilter;

public class GetForbiddenHost {

	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		// TODO Auto-generated method stub

		BufferedReader br = new BufferedReader(new FileReader("forbidden_host"));
		BufferedWriter bw = new BufferedWriter(new FileWriter("forbidden_host_out"));
		String line;
		Set<String> hosts = new HashSet<String>();
		while ( (line = br.readLine()) != null) {
			URL u = new URL(line);
			hosts.add(u.getHost());
		}
		br.close();
		
		for (String s : hosts) {
			bw.write(s);
			bw.write("\n");
		}
		bw.flush();
		bw.close();
	}

}
